package main

import (
	"crawler_distributied/engine"
	"crawler_distributied/persist/client"
	"crawler_distributied/rpcsupport"
	"crawler_distributied/scheduler"
	workerclient "crawler_distributied/worker/client"
	"crawler_distributied/zhenai/parser"
	"flag"
	"log"
	"net/rpc"
	"strings"
)

func createClientPool(hosts []string) chan *rpc.Client {
	var clients []*rpc.Client
	for _, h := range hosts {
		client, err := rpcsupport.NewRpcClient(h)
		if err == nil {
			clients = append(clients, client)
			log.Printf("Connected to %s", h)
		} else {
			log.Printf("Error connecting to %s: %v", h, err)
		}
	}

	out := make(chan *rpc.Client)
	go func() {
		for {
			for _, client := range clients {
				out <- client
			}
		}
	}()
	return out
}

var (
	itemSaverHost = flag.String("itemsaver_host", "", "itemsaver host")
	workerHosts   = flag.String("worker_hosts", "", "worker hosts (comma separated)")
)

func main() {
	flag.Parse()

	// 先初始化存储器
	//itemChan, err := persist.ItemSaver("zhenai_profile")
	itemChan, err := client.ItemSaver(*itemSaverHost)
	if err != nil {
		panic(err)
	}

	pool := createClientPool(strings.Split(*workerHosts, ","))
	processor := workerclient.CreateProcessor(pool)

	// 因为是指针接收者, 必须要定义一个变量, 只有变量才可以取地址
	e := engine.ConcurrentEngine{
		Scheduler:   &scheduler.QueuedScheduler{},
		WorkerCount: 100,
		ItemChan:    itemChan,
		//RequestProcessor: engine.Worker, //单机版
		RequestProcessor: processor,
	}

	// e.Run(engine.Request{
	// 	Url:    "http://www.zhenai.com/zhenghun",
	// 	Parser: engine.NewFuncParser(parser.ParseCityList, "ParseCityList"),
	// })

	// 只抓取一个城市的数据, 便于调试
	e.Run(engine.Request{
		Url:    "https://www.zhenai.com/zhenghun/wuzhou",
		Parser: engine.NewFuncParser(parser.ParseCity, "ParseCity"),
	})
}
